from scrapy import Request, Spider
from urllib import request
from lxml import etree
import json
import re


def optimize_str(str):
    return str.strip().replace('\n', '').replace('\r', '').replace('</br>', '').replace(',', '，')


class TiebaSpider(Spider):
    name = 'tieba_search'

    def __init__(self):
        with open('keywords.txt', encoding='utf-8') as f:
            self.keyword_list = f.read().split('\n')

    def start_requests(self):
        for keyword in self.keyword_list:
            if len(keyword):
                for page in range(3):
                    yield Request(
                        url='https://tieba.baidu.com/f?kw={}&ie=utf-8&pn={}'.format(
                            request.quote(keyword), str(page * 50)),
                        callback=self.parse_list,
                        meta={'keyword': keyword}
                    )

    def parse_list(self, response):
        html = response.text.replace('<!--', '').replace('-->', '')
        html = etree.HTML(html)
        data_list = html.xpath('//*[@id="thread_list"]/li')
        for data in data_list:
            url = data.xpath('.//a[@class="j_th_tit "]/@href')[0]
            url = 'https://tieba.baidu.com' + url
            yield Request(
                url=url,
                callback=self.parse,
                meta={'keyword': response.meta['keyword']}
            )

    def parse(self, response):
        print('parse:', response.url)
        title = ' '.join(''.join(response.xpath('//h3//text()').extract()).split())
        reply_num = response.xpath('//li[@class="l_reply_num"]/span[2]/text()').extract()[0]

        data_list = response.xpath('//div[@class="p_postlist"]/div')
        other_data = data_list[0].xpath('./@data-field').extract()[0]
        other_data = re.sub('"content":".*?",', '', other_data)
        json_data = json.loads(other_data.replace('&quot;', '\"').replace(' ', ''))

        content = ''.join(data_list[0].xpath('.//cc//text()').extract())
        author_nickname = ''.join(data_list[0].xpath('.//li[@class="d_name"]/a/text()').extract())
        author_level_title = ''.join(data_list[0].xpath('.//div[starts-with(@class,"d_badge_title")]/text()').extract())
        author_level = ''.join(data_list[0].xpath('.//div[@class="d_badge_lv"]/text()').extract())
        postdate = ''.join(data_list[0].xpath('.//div[@class="post-tail-wrap"]/span[last()]/text()').extract())
        client_device = ''.join(data_list[0].xpath('.//div[@class="post-tail-wrap"]/span[2]//text()').extract())
        if '来自' in client_device:
            client_device = client_device.replace('来自', '')
        else:
            client_device = ''

        if title == '':
            title = ' '.join(''.join(response.xpath('//h1/text()').extract()).split())
        if postdate == '':
            postdate = json_data['content']['date']
            if 'open_type' in json_data['content']:
                client_device = json_data['content']['open_type']
            else:
                client_device = '手机贴吧'

        if client_device == 'android':
            client_device = 'Android客户端'
        elif client_device == 'apple':
            client_device = 'iPhone客户端'
        elif client_device == 'phone' or client_device == '1':
            client_device = '手机贴吧'
        elif client_device == '0':
            client_device = ''

        author_name = json_data['author']['user_name']
        if 'user_id' in json_data['author']:
            author_id = json_data['author']['user_id']
        else:
            author_id = ''
        post_id = json_data['content']['post_id']

        yield {
            'keyword': response.meta['keyword'],
            'url': response.url,
            'title': optimize_str(title),
            'content': optimize_str(content),
            'post_id': post_id,
            'author_name': author_name,
            'author_nickname': author_nickname,
            'author_id': author_id,
            'author_level_title': author_level_title,
            'author_level': author_level,
            'reply_num': reply_num,
            'postdate': postdate.replace('-', '')[:8],
            'client_device': client_device
        }
